from selenium import webdriver
import time
from bs4 import BeautifulSoup
import logging as log
import datetime
import multiprocess_jd
from pymongo import MongoClient

mongodb_ip = '172.29.13.121'
mongodb_port = 27017
mongodb_lib = 'spider'
mongodb_user = 'spider'
mongodb_pwd = 'spiderQWwe741'



def search_by_url(logo, url):
    print(datetime.datetime.today(), logo, '进入jd.search_by_url')
    db = MongoClient(mongodb_ip, mongodb_port).get_database(mongodb_lib)
    db.authenticate(mongodb_user, mongodb_pwd)
    collection = db.jingdong_phone
    print(datetime.datetime.today(), logo, '数据库连接成功')
    # 初始化配置根据自己chromedriver位置做相应的修改
    from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
    dcap = dict(DesiredCapabilities.PHANTOMJS)
    dcap["phantomjs.page.settings.userAgent"] = (
        "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0")
    # 设置user-agent请求头
    dcap["phantomjs.page.settings.loadImages"] = False  # 禁止加载图片
    print(datetime.datetime.today(), logo, '准备创建driver')
    driver = webdriver.PhantomJS(desired_capabilities=dcap, service_args=multiprocess_jd.service_args)
    driver.implicitly_wait(30)
    driver.set_page_load_timeout(30)
    print(datetime.datetime.today(), logo, 'driver成功创建，开始请求第一页')
    driver.get(url=url)
    print(datetime.datetime.today(), logo, '第一页请求成功')
    time.sleep(10)
    print(datetime.datetime.today(), logo, '第一页拖到底')
    scroll(driver=driver)
    print(datetime.datetime.today(), logo, '第一页拖到底成功')
    time.sleep(10)
    scroll(driver=driver)
    time.sleep(10)
    first_html = driver.page_source
    print(datetime.datetime.today(), logo, '开始提取第一页')
    extract(logo=logo, html=first_html, collection=collection)
    # 检查是否需要翻页
    soup = BeautifulSoup(first_html, 'lxml')
    result_sum = soup.select_one('span#J_resCount').text
    print(datetime.datetime.today(), logo, '查询结果总数:%s' % (result_sum))
    current_page = soup.select_one('div#J_topPage span.fp-text b').text
    print(datetime.datetime.today(), logo, '当前页:%s' % (current_page))
    total_page = soup.select_one('div#J_topPage span.fp-text i').text
    total_page = int(total_page)
    print(datetime.datetime.today(), logo, '总页数：%d' % (total_page))

    if total_page > 1:
        for i in range(2, total_page + 1):
            print(datetime.datetime.today(), logo, '翻页，第%d页' % (i))
            driver.find_element_by_css_selector('div#J_topPage a.fp-next').click()
            time.sleep(3)
            scroll(driver=driver)
            time.sleep(10)
            html = driver.page_source
            extract(logo=logo, html=html, collection=collection)
    print(logo, 'driver关闭')
    driver.quit()


def extract(logo, html, collection):
    soup = BeautifulSoup(html, 'lxml')
    list_data = []
    # 1,搜索正框结果
    for item in soup.select('li.gl-item'):
        try:
            # 手机名称
            name = item.select_one('div.p-name em').text
            # 价格
            price = item.select_one('div.p-price i').text
            # 评论
            commit = item.select_one('div.p-commit strong a').text
            # 店铺
            shop = item.select_one('div.p-shop').text.replace('\n', '')
            # 标签
            icon = item.select_one('div.p-icons').text.replace('\n', '')
            # 链接
            url = 'https:' + item.select_one('div.p-name a')['href']

            single_data = {}
            single_data['name'] = name
            single_data['price'] = price
            single_data['commit'] = commit
            single_data['shop'] = shop
            single_data['icon'] = icon
            single_data['url'] = url
            single_data['logo'] = logo
            now = time.localtime()
            single_data['crawler_time'] = datetime.datetime(now[0], now[1], now[2], now[3], now[4])
            list_data.append(single_data)
        except (AttributeError, TypeError) as e:
            log.exception(e)

    collection.insert(list_data)
    print(datetime.datetime.today(), logo, ' 入库成功，数量：', len(list_data))


# 拖动滚动条到底部
def scroll(driver):
    driver.execute_script("""   
        (function () {   
            var y = document.body.scrollTop;   
            var step = 100;   
            window.scroll(0, y);   


            function f() {   
                if (y < document.body.scrollHeight) {   
                    y += step;   
                    window.scroll(0, y);   
                    setTimeout(f, 50);   
                }  
                else {   
                    window.scroll(0, y);   
                    document.title += "scroll-done";   
                }   
            }   


            setTimeout(f, 1000);   
        })();   
        """)
